#coding:utf8

import pandas as pd
from pyspark.sql import SparkSession
from pyspark.sql.types import StructType, StringType, IntegerType

# 读取csv文件创建DF
if __name__ == '__main__':
    # 构建SparkSession执行环境入口对象
    spark = SparkSession.builder.\
        appName("test_csv").\
        master("local[*]").\
        getOrCreate()

    schema = StructType().add('name', StringType(), nullable=False). \
        add('age', IntegerType(), nullable=False). \
        add('job', StringType(), nullable=False)

    # json可以识别String类型和数字类型
    df = spark.read.format("csv").\
        option("sep",";").\
        option("header",True). \
        option("coding", 'utf8').\
        schema("name STRING,age INT,job STRING").\
        load('../data/input/sql/people.csv')

    df.show()